package com.chance.cc.crawler.development.bootstrap.weibo;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpConstant;
import com.chance.cc.crawler.core.downloader.HttpRequestBody;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import com.chance.cc.crawler.development.utils.RedisReader;
import org.apache.commons.lang3.StringUtils;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRecordFilter.key;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;

/**
 * @Author Zhao.Hhuan
 * @Date Create in 2021/1/8 15:00
 * @Description
 *      微博帖子
 **/
public class WeiboPostStart {

//    http://weibo.com/6594825499/Jmip4cLi1
//    http://weibo.com/5713045949/JmCziAtmW 需要用户验证，信息存在问题
    private static final String testUrl = "http://weibo.com/1640571365/JAdEajRPP";

    public static final String domainId = "weibo";
    public static final String site = "post";

    private static String service_ip = "192.168.1.215";
    private static int service_port = 9599;

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    public static void main(String[] args) {
//        sycMysqlToRedis();
  //      testOneRecord();
        test();
//        testRecord();
    }

    private static void test(){
//        String testUrl = "http://weibo.com/6673392642/JBbFR21G7";
        String testUrl = "https://weibo.com/6405308384/KyrnXa0Ci";
//        String testUrl = "https://weibo.com/aj/v6/comment/big?ajwvr=6&id=4606283939061236&filter=all&from=singleWeiBo";
                          //https://weibo.com/ajax/statuses/buildComments?is_reload=1&id=4682084252385654&is_show_bulletin=2&is_mix=0&count=20&uid=6405308384
        String siteBiz = "post";
        HttpConfig me = HttpConfig.me(domainId);
        me.setHttpSite("post");
        me.setDisableCookie(true);
        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domainId, turnPageItem)
                .httpUrl(testUrl)
                .releaseTime(System.currentTimeMillis())
//                .needWashed(false)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
//                .proxy(proxy)
                .filter(key)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-","filter",domainId,site,siteBiz,"queue")))
//                .addFilterInfo(FilterUtils.dateRangeFilterInfo(0,new long[]{1609430400000L,System.currentTimeMillis()}))
                .build();
        requestRecord.tagsCreator().bizTags().addDomain(domainId);
        requestRecord.tagsCreator().bizTags().addSite(site);
        requestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);
//        requestRecord.getHttpRequest().addExtra("articleUrl","https://weibo.com/5688177048/K7zsif7zE");

        CrawlerRecord commentFilter = new CrawlerRequestRecord();
        commentFilter.setFilter(key);
        commentFilter.addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-","filter",domainId,site,siteBiz,"queue")));
        requestRecord.tagsCreator().resultTags().getCategoryTag().addKVTag("comment_filter_record", JSON.toJSONString(commentFilter));

        DevCrawlerController.builder()
                .triggerInfo(domainId,domainId,System.currentTimeMillis(),domainId)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domainId))
                .consoleResultPipeline()
                .requestRecord(requestRecord)
                .build("com.chance.cc.crawler.development.scripts.weibo").start();

    }

    private static void sycMysqlToRedis(){
        String url = "htps://c.api.weibo.com/2/statuses/count/biz.jsont";
        String site = "subscribeUserId";
        String siteBiz = "dingyue_userId-simple";
        final String source = "1428199813";
        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domainId, CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domainId)
                .httpUrl(url)
                .httpConfig(HttpConfig.me(domainId))
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(2,null))
                .needParsed(false)
                .releaseTime(System.currentTimeMillis())
                .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                .proxy(proxy)
                .build();
        crawlerRequestRecord.setDownload(false);
        crawlerRequestRecord.setSkipPipeline(true);
        crawlerRequestRecord.tagsCreator().bizTags().addDomain(domainId);
        crawlerRequestRecord.tagsCreator().bizTags().addSite(site);
        crawlerRequestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);
        crawlerRequestRecord.getHttpRequest().addExtra("source",source);

        //初始化postRecord
        String requestQueueName = StringUtils.joinWith("-","crawler",domainId,site,siteBiz,"queue");
        CrawlerRequestRecord initCommentCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("weibo_post",turnPageItem)
                .httpUrl("http://"+service_ip+":"+service_port+"/crawler/domain/common/api/v1/"
                        +domainId+"/results/queue/init?requestQueueName="+requestQueueName + "&site=dingyue_userId_post&hourFromNow=24")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();
        HttpRequestBody jsonBody = HttpRequestBody.json(JSON.toJSONString(crawlerRequestRecord), "utf-8");
        initCommentCrawlerRecord.getHttpRequest().setMethod(HttpConstant.Method.POST);
        initCommentCrawlerRecord.getHttpRequest().setRequestBody(jsonBody);

        DevCrawlerController.builder()
                .triggerInfo(domainId,domainId,System.currentTimeMillis(),domainId)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(requestQueueName))
                .consoleResultPipeline()
                .requestRecord(crawlerRequestRecord)
                .supportRecord(initCommentCrawlerRecord)
                .build("com.chance.cc.crawler.development.scripts.weibo").start();

    }

    private static void testOneRecord(){
        String site = "subscribeUserId";
        String requestQueueName = StringUtils.joinWith("-","crawler",domainId,site,"queue");

        //user oauth info source
        CrawlerRequestRecord userOauthInfoCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("weibo_user_oauth_infos",turnPageItem)
                .httpUrl("http://192.168.1.215:9599/crawler/oauth/api/v1/"+domainId+"/userOauthInfos")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        //user agents
        CrawlerRequestRecord userAgentsCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("weibo_user_agents", turnPageItem)
                .httpUrl("https://fake-useragent.herokuapp.com/browsers/0.1.11")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        RedisReader redisReader = new RedisReader("192.168.1.214",6379,3);

        DevCrawlerController.DevCrawlerControllerBuilder devCrawlerControllerBuilder = DevCrawlerController.builder()
//                .crawlerThreadNum(20)
                .triggerInfo(domainId, domainId, System.currentTimeMillis(), domainId)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(requestQueueName))
                .consoleResultPipeline();
//                .supportRecord(userAgentsCrawlerRecord)
//                .supportRecord(userOauthInfoCrawlerRecord);
        for (int i = 0; i < 1; i++) {
            CrawlerRequestRecord crawlerRequestRecord = redisReader.recordPop("crawler-weibo-subscribeUserId-dingyue_userId-simple-queue_list",
                    "crawler-weibo-subscribeUserId-dingyue_userId-simple-queue_map");
            devCrawlerControllerBuilder
                    .requestRecord(crawlerRequestRecord);
        }

        devCrawlerControllerBuilder.build().start();

    }

    private static void testRecord(){
        String s = "{\"collectionTime\":1620284184885,\"domain\":\"weibo\",\"download\":true,\"endTimeFromSchedulerTime\":true,\"filterPipelineResult\":false,\"httpConfig\":{\"circularRedirectsAllowed\":false,\"connectionRequestTimeout\":-1,\"connectionTimeout\":30000,\"disableCookie\":false,\"httpDomain\":\"weibo\",\"httpSite\":\"default-site\",\"proxy\":{\"host\":\"http-dyn.abuyun.com\",\"password\":\"71F33D94CE5F7BF2\",\"port\":9020,\"username\":\"HL89Q19E86E2987D\"},\"requestRetryTimes\":0,\"responseTextGenerateHtml\":true,\"socketTimeout\":20000,\"useGzip\":true,\"useProxy\":true,\"userAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36\"},\"httpRequest\":{\"binaryContent\":false,\"cookies\":{},\"headers\":{},\"url\":\"https://c.api.weibo.com/2/statuses/count/biz.json?access_token=2.00o4_w1HrAaeYBedf38e38b8SnITmD&ids=4633855904192108\"},\"needParsedPage\":true,\"needWashPage\":true,\"recordKey\":\"https://c.api.weibo.com/2/statuses/count/biz.json?access_token=2.00o4_w1HrAaeYBedf38e38b8SnITmD&ids=4633855904192108\",\"releaseTime\":1620280803000,\"scheduleTime\":1620284180376,\"skipPipeline\":false,\"tags\":{\"crawler_request_tag\":{\"category\":\"crawler_request_tag\",\"labelTags\":{\"turn_page_item_request\":{\"label\":\"turn_page_item_request\"}}},\"crawler_schedule_tag\":{\"category\":\"crawler_schedule_tag\",\"kvTags\":{\"Schedule_Trigger_Time\":{\"key\":\"Schedule_Trigger_Time\",\"val\":1620284180376},\"Schedule_Trigger_Job_Key\":{\"key\":\"Schedule_Trigger_Job_Key\",\"val\":\"weibo-subscribeUserId-dingyue_userId-simple-realtime\"},\"Schedule_Trigger_Key\":{\"key\":\"Schedule_Trigger_Key\",\"val\":\"weibo-subscribeUserId-dingyue_userId-simple-realtime-simple\"}}},\"crawler_business_tag\":{\"category\":\"crawler_business_tag\",\"kvTags\":{\"site\":{\"key\":\"site\",\"val\":\"subscribeUserId\"},\"domain\":{\"key\":\"domain\",\"val\":\"weibo\"},\"site_biz\":{\"key\":\"site_biz\",\"val\":\"dingyue_userId-simple\"}}},\"crawler_result_tag\":{\"category\":\"crawler_result_tag\",\"labelTags\":{\"interaction\":{\"label\":\"interaction\"}}}}}";
        CrawlerRequestRecord crawlerRequestRecord = JSONObject.parseObject(s,CrawlerRequestRecord.class);

        DevCrawlerController.builder()
                .triggerInfo(domainId, domainId, System.currentTimeMillis(), domainId)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domainId))
                .consoleResultPipeline()
                .requestRecord(crawlerRequestRecord)
                .build()
                .start();
    }

}
